{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cdf841d-bc87-4138-96d6-f8b6b173ddd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup OpenAI Agent\n",
    "import openai\n",
    "\n",
    "openai.api_key = \"sk-your-key\"\n",
    "from llama_index.agent import OpenAIAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6358bd48-627c-47d6-9761-376877195950",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Calling Function ===\n",
      "Calling function: text_to_speech with args: {\n",
      "  \"text\": \"hello world\"\n",
      "}\n",
      "Got output: Audio playback complete.\n",
      "========================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from llama_index.tools.azure_speech.base import AzureSpeechToolSpec\n",
    "from llama_index.tools.azure_translate.base import AzureTranslateToolSpec\n",
    "\n",
    "speech_tool = AzureSpeechToolSpec(speech_key=\"your-key\", region=\"eastus\")\n",
    "translate_tool = AzureTranslateToolSpec(api_key=\"your-key\", region=\"eastus\")\n",
    "\n",
    "agent = OpenAIAgent.from_tools(\n",
    "    [*speech_tool.to_tool_list(), *translate_tool.to_tool_list()],\n",
    "    verbose=True,\n",
    ")\n",
    "print(agent.chat('Say \"hello world\"'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfd64436-ec97-4fa5-977a-0e520ccc7e0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('data/speech.wav', <http.client.HTTPMessage at 0x7f8cf3c84fa0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We use one of the Azure resources of an insurance call for the next example.\n",
    "# Use this cell to download the file, or put your own file in data/speech.wav\n",
    "import urllib.request\n",
    "\n",
    "urllib.request.urlretrieve(\n",
    "    \"https://speechstudiorawgithubscenarioscdn.azureedge.net/call-center/sampledata/Call1_separated_16k_health_insurance.wav\",\n",
    "    \"data/speech.wav\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71c0244c-d9f3-4849-8571-50cee19f705b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Calling Function ===\n",
      "Calling function: speech_to_text with args: {\n",
      "  \"filename\": \"data/speech.wav\"\n",
      "}\n",
      "Got output: ['Hello, thank you for calling Contoso. Who am I speaking with today?', \"Hi, my name is Mary Rondo. I'm trying to enroll myself with conscious So.\", 'Hi, Mary. Uh, are you calling because you need health insurance?', \"Yes, yeah, I'm calling to sign up for insurance.\", 'Great. Uh, if you can answer a few questions, uh, we can get you signed up in the jiffy.', 'OK.', \"So uh, what's your full name?\", 'Uh, so Mary Beth Rondo, last name is R like Romeo, O like Ocean, and like Nancy. DD like Dog and O like Ocean again.', \"Rando got it. And what's the best callback number in case we get disconnected?\", 'I only have a cell phone, so I can give you that.', \"Yeah, that'll be fine.\", \"Sure. So it's 234554 and then 9312.\", \"To confirm, it's 234-554-9312.\", \"Yep, that's right.\", \"Excellent. Uh, Let's get some additional information from your app. For your application, Uh, do you have a job?\", 'Uh, yes, I am self-employed.', 'OK, So then you have a Social Security number as well? Uh, yes, I do.', 'OK, uh, and what is your Social Security number please?', \"Uh, sure. So it's 412.\", 'Uh 256789.', 'Sorry, what was that, A-25 or A225 you cut out for a bit?', \"Uh, it's 22 so.\", '412, then another two, then five.', 'Hey, thank you so much and could I have your e-mail address please?', \"Yeah, it's Mary rondo@gmail.com, so myfirstandlastname@gmail.com. No periods, no dashes.\", \"Great. Uh, that is the last question. So let me take your information and I'll be able to get you signed up right away. Thank you for calling Contoso and I'll be able to get you signed up immediately. One of our agents will call you back in about 24 hours or so to confirm your application.\", 'That sounds great. Thank you.', 'Absolutely. If you need anything else, please give us a call at 1-800-555-5564 ext 123. Thank you very much for calling Contoso.', 'Uh, actually uh, sorry, one more question.', 'Ohh yes, of course.', \"I'm curious what I'd be getting a physical card as proof of coverage.\", 'So the default is a digital membership card, but we can send you a physical card if you prefer.', \"Uh, yes. Could you please mail it to me when it's ready? I'd like to have it shipped to you from my address.\", 'Ohh yeah.', \"Uh so it's 2660 Unit A on Maple Ave. SE Lansing and then zip code is 48823.\", \"Absolutely. I've made a note on your file.\", 'Awesome. Thanks so much.', \"You're very welcome. Thank you for calling Contoso and have a great day.\"]\n",
      "========================\n",
      "Here is the transcribed conversation:\n",
      "\n",
      "1. Hello, thank you for calling Contoso. Who am I speaking with today?\n",
      "2. Hi, my name is Mary Rondo. I'm trying to enroll myself with Contoso.\n",
      "3. Hi, Mary. Are you calling because you need health insurance?\n",
      "4. Yes, I'm calling to sign up for insurance.\n",
      "5. Great. If you can answer a few questions, we can get you signed up in a jiffy.\n",
      "6. OK.\n",
      "7. So, what's your full name?\n",
      "8. Mary Beth Rondo, last name is R like Romeo, O like Ocean, N like Nancy, D like Dog, and O like Ocean again.\n",
      "9. Got it. And what's the best callback number in case we get disconnected?\n",
      "10. I only have a cell phone, so I can give you that.\n",
      "11. Yeah, that'll be fine.\n",
      "12. Sure. So it's 234-554-9312.\n",
      "13. To confirm, it's 234-554-9312.\n",
      "14. Yep, that's right.\n",
      "15. Excellent. Let's get some additional information from your app. For your application, do you have a job?\n",
      "16. Yes, I am self-employed.\n",
      "17. OK, so then you have a Social Security number as well?\n",
      "18. Yes, I do.\n",
      "19. OK, and what is your Social Security number, please?\n",
      "20. Sure. So it's 412-256789.\n",
      "21. Sorry, what was that? A-25 or A225? You cut out for a bit.\n",
      "22. It's 22, so 412-2256789.\n",
      "23. Thank you so much. And could I have your email address, please?\n",
      "24. Yeah, it's MaryRondo@gmail.com, so myfirstandlastname@gmail.com. No periods, no dashes.\n",
      "25. Great. That is the last question. Let me take your information, and I'll be able to get you signed up right away. Thank you for calling Contoso, and I'll be able to get you signed up immediately. One of our agents will call you back in about 24 hours or so to confirm your application.\n",
      "26. That sounds great. Thank you.\n",
      "27. Absolutely. If you need anything else, please give us a call at 1-800-555-5564 ext 123. Thank you very much for calling Contoso.\n",
      "28. Actually, sorry, one more question.\n",
      "29. Oh, yes, of course.\n",
      "30. I'm curious, will I be getting a physical card as proof of coverage?\n",
      "31. So the default is a digital membership card, but we can send you a physical card if you prefer.\n",
      "32. Yes, could you please mail it to me when it's ready? I'd like to have it shipped to my address.\n",
      "33. Absolutely. I've made a note on your file.\n",
      "34. Awesome. Thanks so much.\n",
      "35. You're very welcome. Thank you for calling Contoso and have a great day.\n"
     ]
    }
   ],
   "source": [
    "print(agent.chat(\"transcribe and format conversation in data/speech.wav\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dc5d831-f08d-4d62-993a-f1b6e4070f7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Calling Function ===\n",
      "Calling function: translate with args: {\n",
      "  \"text\": \"Hello, thank you for calling Contoso. Who am I speaking with today? Hi, my name is Mary Rondo. I'm trying to enroll myself with Contoso. Hi, Mary. Are you calling because you need health insurance? Yes, I'm calling to sign up for insurance. Great. If you can answer a few questions, we can get you signed up in a jiffy. OK. So, what's your full name? Mary Beth Rondo, last name is R like Romeo, O like Ocean, N like Nancy, D like Dog, and O like Ocean again. Got it. And what's the best callback number in case we get disconnected? I only have a cell phone, so I can give you that. Yeah, that'll be fine. Sure. So it's 234-554-9312. To confirm, it's 234-554-9312. Yep, that's right. Excellent. Let's get some additional information from your app. For your application, do you have a job? Yes, I am self-employed. OK, so then you have a Social Security number as well? Yes, I do. OK, and what is your Social Security number, please? Sure. So it's 412-256789. Sorry, what was that? A-25 or A225? You cut out for a bit. It's 22, so 412-2256789. Thank you so much. And could I have your email address, please? Yeah, it's MaryRondo@gmail.com, so myfirstandlastname@gmail.com. No periods, no dashes. Great. That is the last question. Let me take your information, and I'll be able to get you signed up right away. Thank you for calling Contoso, and I'll be able to get you signed up immediately. One of our agents will call you back in about 24 hours or so to confirm your application. That sounds great. Thank you. Absolutely. If you need anything else, please give us a call at 1-800-555-5564 ext 123. Thank you very much for calling Contoso. Actually, sorry, one more question. Oh, yes, of course. I'm curious, will I be getting a physical card as proof of coverage? So the default is a digital membership card, but we can send you a physical card if you prefer. Yes, could you please mail it to me when it's ready? I'd like to have it shipped to my address. Absolutely. I've made a note on your file. Awesome. Thanks so much. You're very welcome. Thank you for calling Contoso and have a great day.\",\n",
      "  \"language\": \"es\"\n",
      "}\n",
      "Got output: [{'detectedLanguage': {'language': 'en', 'score': 1.0}, 'translations': [{'text': 'Hola, gracias por llamar a Contoso. ¿Con quién estoy hablando hoy? Hola, mi nombre es Mary Rondo. Estoy intentando inscribirme en Contoso. Hola, María. ¿Está llamando porque necesita seguro médico? Sí, estoy llamando para inscribirme en un seguro. Bien. Si puede responder algunas preguntas, podemos hacer que se registre en un santiamén. De acuerdo. Entonces, ¿cuál es tu nombre completo? Mary Beth Rondo, el apellido es R como Romeo, O como Ocean, N como Nancy, D como Dog y O como Ocean nuevamente. Entiendo. ¿Y cuál es el mejor número de devolución de llamada en caso de que nos desconectemos? Solo tengo un teléfono celular, así que puedo dártelo. Sí, eso estará bien. Seguro. Así que es 234-554-9312. Para confirmar, es 234-554-9312. Sí, así es. Excelente. Obtengamos información adicional de tu aplicación. Para su solicitud, ¿tiene un trabajo? Sí, trabajo por cuenta propia. Bien, ¿entonces también tienes un número de Seguro Social? Sí, lo hago. OK, ¿y cuál es su número de Seguro Social, por favor? Seguro. Así que es 412-256789. Lo siento, ¿qué fue eso? ¿A-25 o A225? Cortas un poco. Son 22, así que 412-2256789. Muchas gracias. ¿Y podría tener su dirección de correo electrónico, por favor? Sí, es MaryRondo@gmail.com, así que myfirstandlastname@gmail.com. Sin puntos, sin guiones. Bien. Esa es la última pregunta. Déjame tomar tu información, y podré inscribirte de inmediato. Gracias por llamar a Contoso y podré registrarte inmediatamente. Uno de nuestros agentes le devolverá la llamada en aproximadamente 24 horas para confirmar su solicitud. Eso es excelente. Gracias. Absolutamente. Si necesita algo más, llámenos al 1-800-555-5564 ext 123. Muchas gracias por llamar a Contoso. En realidad, perdón, una pregunta más. Oh, sí, por supuesto. Tengo curiosidad, ¿recibiré una tarjeta física como prueba de cobertura? Por lo tanto, el valor predeterminado es una tarjeta de membresía digital, pero podemos enviarle una tarjeta física si lo prefiere. Sí, ¿podría enviármelo por correo cuando esté listo? Me gustaría que me lo enviaran a mi dirección. Absolutamente. He tomado nota en su archivo. Genial. Muchas gracias. De nada. Gracias por llamar a Contoso y que tengas un gran día.', 'to': 'es'}]}]\n",
      "========================\n",
      "Aquí está la conversación traducida al español:\n",
      "\n",
      "1. Hola, gracias por llamar a Contoso. ¿Con quién estoy hablando hoy?\n",
      "2. Hola, mi nombre es Mary Rondo. Estoy intentando inscribirme en Contoso.\n",
      "3. Hola, María. ¿Está llamando porque necesita seguro médico?\n",
      "4. Sí, estoy llamando para inscribirme en un seguro.\n",
      "5. Bien. Si puede responder algunas preguntas, podemos hacer que se registre en un santiamén.\n",
      "6. De acuerdo.\n",
      "7. Entonces, ¿cuál es tu nombre completo?\n",
      "8. Mary Beth Rondo, el apellido es R como Romeo, O como Ocean, N como Nancy, D como Dog y O como Ocean nuevamente.\n",
      "9. Entiendo. ¿Y cuál es el mejor número de devolución de llamada en caso de que nos desconectemos?\n",
      "10. Solo tengo un teléfono celular, así que puedo dártelo.\n",
      "11. Sí, eso estará bien.\n",
      "12. Seguro. Así que es 234-554-9312.\n",
      "13. Para confirmar, es 234-554-9312.\n",
      "14. Sí, así es.\n",
      "15. Excelente. Obtengamos información adicional de tu aplicación. Para su solicitud, ¿tiene un trabajo?\n",
      "16. Sí, trabajo por cuenta propia.\n",
      "17. Bien, ¿entonces también tienes un número de Seguro Social?\n",
      "18. Sí, lo hago.\n",
      "19. OK, ¿y cuál es su número de Seguro Social, por favor?\n",
      "20. Seguro. Así que es 412-256789.\n",
      "21. Lo siento, ¿qué fue eso? ¿A-25 o A225? Cortas un poco.\n",
      "22. Son 22, así que 412-2256789.\n",
      "23. Muchas gracias. ¿Y podría tener su dirección de correo electrónico, por favor?\n",
      "24. Sí, es MaryRondo@gmail.com, así que myfirstandlastname@gmail.com. Sin puntos, sin guiones.\n",
      "25. Bien. Esa es la última pregunta. Déjame tomar tu información, y podré inscribirte de inmediato. Gracias por llamar a Contoso y podré registrarte inmediatamente. Uno de nuestros agentes le devolverá la llamada en aproximadamente 24 horas para confirmar su solicitud.\n",
      "26. Eso es excelente. Gracias.\n",
      "27. Absolutamente. Si necesita algo más, llámenos al 1-800-555-5564 ext 123. Muchas gracias por llamar a Contoso.\n",
      "28. En realidad, perdón, una pregunta más.\n",
      "29. Oh, sí, por supuesto.\n",
      "30. Tengo curiosidad, ¿recibiré una tarjeta física como prueba de cobertura?\n",
      "31. Por lo tanto, el valor predeterminado es una tarjeta de membresía digital, pero podemos enviarle una tarjeta física si lo prefiere.\n",
      "32. Sí, ¿podría enviármelo por correo cuando esté listo? Me gustaría que me lo enviaran a mi dirección.\n",
      "33. Absolutamente. He tomado nota en su archivo.\n",
      "34. Genial. Muchas gracias.\n",
      "35. De nada. Gracias por llamar a Contoso y que tengas un gran día.\n"
     ]
    }
   ],
   "source": [
    "print(agent.chat(\"translate the conversation into spanish\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
